{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use DDPG to Play Pendulum-v0\n",
    "\n",
    "PyTorch version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "import copy\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "torch.manual_seed(0)\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:50:01 [INFO] env: <PendulumEnv<Pendulum-v0>>\n",
      "22:50:01 [INFO] action_space: Box(-2.0, 2.0, (1,), float32)\n",
      "22:50:01 [INFO] observation_space: Box(-8.0, 8.0, (3,), float32)\n",
      "22:50:01 [INFO] reward_range: (-inf, inf)\n",
      "22:50:01 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 30}\n",
      "22:50:01 [INFO] _max_episode_steps: 200\n",
      "22:50:01 [INFO] _elapsed_steps: None\n",
      "22:50:01 [INFO] id: Pendulum-v0\n",
      "22:50:01 [INFO] entry_point: gym.envs.classic_control:PendulumEnv\n",
      "22:50:01 [INFO] reward_threshold: None\n",
      "22:50:01 [INFO] nondeterministic: False\n",
      "22:50:01 [INFO] max_episode_steps: 200\n",
      "22:50:01 [INFO] _kwargs: {}\n",
      "22:50:01 [INFO] _env_name: Pendulum\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Pendulum-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNReplayer:\n",
    "    def __init__(self, capacity):\n",
    "        self.memory = pd.DataFrame(index=range(capacity),\n",
    "                columns=['observation', 'action', 'reward',\n",
    "                'next_observation', 'done'])\n",
    "        self.i = 0\n",
    "        self.count = 0\n",
    "        self.capacity = capacity\n",
    "\n",
    "    def store(self, *args):\n",
    "        self.memory.loc[self.i] = args\n",
    "        self.i = (self.i + 1) % self.capacity\n",
    "        self.count = min(self.count + 1, self.capacity)\n",
    "\n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.count, size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OrnsteinUhlenbeckProcess:\n",
    "    def __init__(self, x0):\n",
    "        self.x = x0\n",
    "\n",
    "    def __call__(self, mu=0., sigma=1., theta=.15, dt=.01):\n",
    "        n = np.random.normal(size=self.x.shape)\n",
    "        self.x += (theta * (mu - self.x) * dt + sigma * np.sqrt(dt) * n)\n",
    "        return self.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DDPGAgent:\n",
    "    def __init__(self, env):\n",
    "        state_dim = env.observation_space.shape[0]\n",
    "        self.action_dim = env.action_space.shape[0]\n",
    "        self.action_low = env.action_space.low[0]\n",
    "        self.action_high = env.action_space.high[0]\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.replayer = DQNReplayer(20000)\n",
    "\n",
    "        self.actor_evaluate_net = self.build_net(\n",
    "                input_size=state_dim, hidden_sizes=[32, 64],\n",
    "                output_size=self.action_dim)\n",
    "        self.actor_optimizer = optim.Adam(self.actor_evaluate_net.parameters(), lr=0.0001)\n",
    "        self.actor_target_net = copy.deepcopy(self.actor_evaluate_net)\n",
    "\n",
    "        self.critic_evaluate_net = self.build_net(\n",
    "                input_size=state_dim+self.action_dim, hidden_sizes=[64, 128])\n",
    "        self.critic_optimizer = optim.Adam(self.critic_evaluate_net.parameters(), lr=0.001)\n",
    "        self.critic_loss = nn.MSELoss()\n",
    "        self.critic_target_net = copy.deepcopy(self.critic_evaluate_net)\n",
    "\n",
    "    def build_net(self, input_size, hidden_sizes, output_size=1,\n",
    "            output_activator=None):\n",
    "        layers = []\n",
    "        for input_size, output_size in zip(\n",
    "                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):\n",
    "            layers.append(nn.Linear(input_size, output_size))\n",
    "            layers.append(nn.ReLU())\n",
    "        layers = layers[:-1]\n",
    "        if output_activator:\n",
    "            layers.append(output_activator)\n",
    "        net = nn.Sequential(*layers)\n",
    "        return net\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.noise = OrnsteinUhlenbeckProcess(np.zeros((self.action_dim,)))\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train' and self.replayer.count < 3000:\n",
    "            action = np.random.uniform(self.action_low, self.action_high)\n",
    "        else:\n",
    "            state_tensor = torch.as_tensor(observation, dtype=torch.float).reshape(1, -1)\n",
    "            action_tensor = self.actor_evaluate_net(state_tensor)\n",
    "            action = action_tensor.detach().numpy()[0]\n",
    "        if self.mode == 'train':\n",
    "            noise = self.noise(sigma=0.1)\n",
    "            action = (action + noise).clip(self.action_low, self.action_high)\n",
    "\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                state, _, _, act, next_state, reward, done, _ = self.trajectory[-8:]\n",
    "                self.replayer.store(state, act, reward, next_state, done)\n",
    "\n",
    "            if self.replayer.count >= 3000:\n",
    "                self.learn()\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def update_net(self, target_net, evaluate_net, learning_rate=0.005):\n",
    "        for target_param, evaluate_param in zip(\n",
    "                target_net.parameters(), evaluate_net.parameters()):\n",
    "            target_param.data.copy_(learning_rate * evaluate_param.data\n",
    "                    + (1 - learning_rate) * target_param.data)\n",
    "\n",
    "    def learn(self):\n",
    "        # replay\n",
    "        states, actions, rewards, next_states, dones = self.replayer.sample(64)\n",
    "        state_tensor = torch.as_tensor(states, dtype=torch.float)\n",
    "        action_tensor = torch.as_tensor(actions, dtype=torch.long)\n",
    "        reward_tensor = torch.as_tensor(rewards, dtype=torch.float)\n",
    "        next_state_tensor = torch.as_tensor(next_states, dtype=torch.float)\n",
    "        done_tensor = torch.as_tensor(dones, dtype=torch.float)\n",
    "\n",
    "        # learn critic\n",
    "        next_action_tensor = self.actor_target_net(next_state_tensor)\n",
    "        noise_tensor = (0.2 * torch.randn_like(action_tensor, dtype=torch.float))\n",
    "        noisy_next_action_tensor = (next_action_tensor + noise_tensor).clamp(\n",
    "                self.action_low, self.action_high)\n",
    "        next_state_action_tensor = torch.cat([next_state_tensor, noisy_next_action_tensor], 1)\n",
    "        next_q_tensor = self.critic_target_net(next_state_action_tensor).squeeze(1)\n",
    "        critic_target_tensor = reward_tensor + (1. - done_tensor) * self.gamma * next_q_tensor\n",
    "        critic_target_tensor = critic_target_tensor.detach()\n",
    "\n",
    "        state_action_tensor = torch.cat([state_tensor, action_tensor], 1)\n",
    "        critic_pred_tensor = self.critic_evaluate_net(state_action_tensor).squeeze(1)\n",
    "        critic_loss_tensor = self.critic_loss(critic_pred_tensor, critic_target_tensor)\n",
    "        self.critic_optimizer.zero_grad()\n",
    "        critic_loss_tensor.backward()\n",
    "        self.critic_optimizer.step()\n",
    "\n",
    "        # learn actor\n",
    "        pred_action_tensor = self.actor_evaluate_net(state_tensor)\n",
    "        pred_action_tensor = pred_action_tensor.clamp(self.action_low, self.action_high)\n",
    "        pred_state_action_tensor = torch.cat([state_tensor, pred_action_tensor], 1)\n",
    "        critic_pred_tensor = self.critic_evaluate_net(pred_state_action_tensor)\n",
    "        actor_loss_tensor = -critic_pred_tensor.mean()\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        actor_loss_tensor.backward()\n",
    "        self.actor_optimizer.step()\n",
    "\n",
    "        self.update_net(self.critic_target_net, self.critic_evaluate_net)\n",
    "        self.update_net(self.actor_target_net, self.actor_evaluate_net)\n",
    "\n",
    "\n",
    "agent = DDPGAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:50:02 [INFO] ==== train ====\n",
      "22:50:02 [DEBUG] train episode 0: reward = -1744.13, steps = 200\n",
      "22:50:02 [DEBUG] train episode 1: reward = -1025.25, steps = 200\n",
      "22:50:02 [DEBUG] train episode 2: reward = -1590.20, steps = 200\n",
      "22:50:02 [DEBUG] train episode 3: reward = -1137.77, steps = 200\n",
      "22:50:02 [DEBUG] train episode 4: reward = -1675.82, steps = 200\n",
      "22:50:02 [DEBUG] train episode 5: reward = -1632.97, steps = 200\n",
      "22:50:02 [DEBUG] train episode 6: reward = -753.85, steps = 200\n",
      "22:50:02 [DEBUG] train episode 7: reward = -1833.66, steps = 200\n",
      "22:50:03 [DEBUG] train episode 8: reward = -936.49, steps = 200\n",
      "22:50:03 [DEBUG] train episode 9: reward = -1622.68, steps = 200\n",
      "22:50:03 [DEBUG] train episode 10: reward = -1307.43, steps = 200\n",
      "22:50:03 [DEBUG] train episode 11: reward = -908.99, steps = 200\n",
      "22:50:03 [DEBUG] train episode 12: reward = -1504.19, steps = 200\n",
      "22:50:03 [DEBUG] train episode 13: reward = -1003.41, steps = 200\n",
      "22:50:03 [DEBUG] train episode 14: reward = -921.67, steps = 200\n",
      "22:50:22 [DEBUG] train episode 15: reward = -1015.46, steps = 200\n",
      "22:50:40 [DEBUG] train episode 16: reward = -1002.62, steps = 200\n",
      "22:50:58 [DEBUG] train episode 17: reward = -1498.86, steps = 200\n",
      "22:51:17 [DEBUG] train episode 18: reward = -1770.69, steps = 200\n",
      "22:51:35 [DEBUG] train episode 19: reward = -1368.10, steps = 200\n",
      "22:51:53 [DEBUG] train episode 20: reward = -1422.33, steps = 200\n",
      "22:52:12 [DEBUG] train episode 21: reward = -1505.02, steps = 200\n",
      "22:52:31 [DEBUG] train episode 22: reward = -1525.87, steps = 200\n",
      "22:52:49 [DEBUG] train episode 23: reward = -1331.44, steps = 200\n",
      "22:53:08 [DEBUG] train episode 24: reward = -1093.97, steps = 200\n",
      "22:53:26 [DEBUG] train episode 25: reward = -1222.73, steps = 200\n",
      "22:53:46 [DEBUG] train episode 26: reward = -961.30, steps = 200\n",
      "22:54:06 [DEBUG] train episode 27: reward = -901.85, steps = 200\n",
      "22:54:25 [DEBUG] train episode 28: reward = -697.18, steps = 200\n",
      "22:54:46 [DEBUG] train episode 29: reward = -904.89, steps = 200\n",
      "22:55:05 [DEBUG] train episode 30: reward = -930.82, steps = 200\n",
      "22:55:24 [DEBUG] train episode 31: reward = -926.90, steps = 200\n",
      "22:55:42 [DEBUG] train episode 32: reward = -790.95, steps = 200\n",
      "22:56:00 [DEBUG] train episode 33: reward = -793.23, steps = 200\n",
      "22:56:19 [DEBUG] train episode 34: reward = -664.05, steps = 200\n",
      "22:56:37 [DEBUG] train episode 35: reward = -796.29, steps = 200\n",
      "22:56:56 [DEBUG] train episode 36: reward = -721.84, steps = 200\n",
      "22:57:14 [DEBUG] train episode 37: reward = -520.92, steps = 200\n",
      "22:57:33 [DEBUG] train episode 38: reward = -516.89, steps = 200\n",
      "22:57:51 [DEBUG] train episode 39: reward = -625.72, steps = 200\n",
      "22:58:10 [DEBUG] train episode 40: reward = -767.70, steps = 200\n",
      "22:58:28 [DEBUG] train episode 41: reward = -755.03, steps = 200\n",
      "22:58:47 [DEBUG] train episode 42: reward = -264.11, steps = 200\n",
      "22:59:06 [DEBUG] train episode 43: reward = -391.49, steps = 200\n",
      "22:59:25 [DEBUG] train episode 44: reward = -517.31, steps = 200\n",
      "22:59:43 [DEBUG] train episode 45: reward = -534.10, steps = 200\n",
      "23:00:03 [DEBUG] train episode 46: reward = -506.09, steps = 200\n",
      "23:00:27 [DEBUG] train episode 47: reward = -385.75, steps = 200\n",
      "23:00:51 [DEBUG] train episode 48: reward = -518.91, steps = 200\n",
      "23:01:15 [DEBUG] train episode 49: reward = -395.98, steps = 200\n",
      "23:01:36 [DEBUG] train episode 50: reward = -626.09, steps = 200\n",
      "23:02:01 [DEBUG] train episode 51: reward = -168.21, steps = 200\n",
      "23:02:20 [DEBUG] train episode 52: reward = -120.23, steps = 200\n",
      "23:02:39 [DEBUG] train episode 53: reward = -256.31, steps = 200\n",
      "23:03:00 [DEBUG] train episode 54: reward = -140.80, steps = 200\n",
      "23:03:25 [DEBUG] train episode 55: reward = -249.16, steps = 200\n",
      "23:03:45 [DEBUG] train episode 56: reward = -265.54, steps = 200\n",
      "23:04:04 [DEBUG] train episode 57: reward = -146.78, steps = 200\n",
      "23:04:24 [DEBUG] train episode 58: reward = -510.25, steps = 200\n",
      "23:04:45 [DEBUG] train episode 59: reward = -399.44, steps = 200\n",
      "23:05:05 [DEBUG] train episode 60: reward = -521.06, steps = 200\n",
      "23:05:25 [DEBUG] train episode 61: reward = -503.14, steps = 200\n",
      "23:05:45 [DEBUG] train episode 62: reward = -500.67, steps = 200\n",
      "23:06:06 [DEBUG] train episode 63: reward = -558.42, steps = 200\n",
      "23:06:25 [DEBUG] train episode 64: reward = -370.34, steps = 200\n",
      "23:06:44 [DEBUG] train episode 65: reward = -383.06, steps = 200\n",
      "23:07:03 [DEBUG] train episode 66: reward = -256.65, steps = 200\n",
      "23:07:22 [DEBUG] train episode 67: reward = -248.25, steps = 200\n",
      "23:07:42 [DEBUG] train episode 68: reward = -364.46, steps = 200\n",
      "23:08:01 [DEBUG] train episode 69: reward = -249.60, steps = 200\n",
      "23:08:21 [DEBUG] train episode 70: reward = -126.43, steps = 200\n",
      "23:08:41 [DEBUG] train episode 71: reward = -151.36, steps = 200\n",
      "23:09:00 [DEBUG] train episode 72: reward = -240.94, steps = 200\n",
      "23:09:20 [DEBUG] train episode 73: reward = -238.82, steps = 200\n",
      "23:09:40 [DEBUG] train episode 74: reward = -374.65, steps = 200\n",
      "23:10:01 [DEBUG] train episode 75: reward = -135.91, steps = 200\n",
      "23:10:22 [DEBUG] train episode 76: reward = -128.82, steps = 200\n",
      "23:10:43 [DEBUG] train episode 77: reward = -259.55, steps = 200\n",
      "23:11:05 [DEBUG] train episode 78: reward = -252.11, steps = 200\n",
      "23:11:30 [DEBUG] train episode 79: reward = -258.99, steps = 200\n",
      "23:11:54 [DEBUG] train episode 80: reward = -409.07, steps = 200\n",
      "23:12:25 [DEBUG] train episode 81: reward = -260.35, steps = 200\n",
      "23:12:58 [DEBUG] train episode 82: reward = -735.68, steps = 200\n",
      "23:13:34 [DEBUG] train episode 83: reward = -129.49, steps = 200\n",
      "23:14:09 [DEBUG] train episode 84: reward = -380.57, steps = 200\n",
      "23:14:42 [DEBUG] train episode 85: reward = -252.26, steps = 200\n",
      "23:15:15 [DEBUG] train episode 86: reward = -482.08, steps = 200\n",
      "23:15:49 [DEBUG] train episode 87: reward = -247.00, steps = 200\n",
      "23:16:25 [DEBUG] train episode 88: reward = -509.94, steps = 200\n",
      "23:16:58 [DEBUG] train episode 89: reward = -124.17, steps = 200\n",
      "23:17:31 [DEBUG] train episode 90: reward = -508.79, steps = 200\n",
      "23:18:05 [DEBUG] train episode 91: reward = -629.38, steps = 200\n",
      "23:18:38 [DEBUG] train episode 92: reward = -252.29, steps = 200\n",
      "23:19:11 [DEBUG] train episode 93: reward = -604.16, steps = 200\n",
      "23:19:44 [DEBUG] train episode 94: reward = -391.61, steps = 200\n",
      "23:20:18 [DEBUG] train episode 95: reward = -794.07, steps = 200\n",
      "23:20:52 [DEBUG] train episode 96: reward = -388.14, steps = 200\n",
      "23:21:26 [DEBUG] train episode 97: reward = -562.29, steps = 200\n",
      "23:22:00 [DEBUG] train episode 98: reward = -382.14, steps = 200\n",
      "23:22:33 [DEBUG] train episode 99: reward = -634.31, steps = 200\n",
      "23:23:07 [DEBUG] train episode 100: reward = -610.10, steps = 200\n",
      "23:23:43 [DEBUG] train episode 101: reward = -627.77, steps = 200\n",
      "23:24:17 [DEBUG] train episode 102: reward = -500.54, steps = 200\n",
      "23:24:50 [DEBUG] train episode 103: reward = -649.02, steps = 200\n",
      "23:25:24 [DEBUG] train episode 104: reward = -506.47, steps = 200\n",
      "23:25:57 [DEBUG] train episode 105: reward = -489.89, steps = 200\n",
      "23:26:35 [DEBUG] train episode 106: reward = -495.22, steps = 200\n",
      "23:27:08 [DEBUG] train episode 107: reward = -620.70, steps = 200\n",
      "23:27:41 [DEBUG] train episode 108: reward = -507.61, steps = 200\n",
      "23:28:15 [DEBUG] train episode 109: reward = -624.60, steps = 200\n",
      "23:28:49 [DEBUG] train episode 110: reward = -385.68, steps = 200\n",
      "23:29:23 [DEBUG] train episode 111: reward = -510.95, steps = 200\n",
      "23:29:57 [DEBUG] train episode 112: reward = -615.34, steps = 200\n",
      "23:30:34 [DEBUG] train episode 113: reward = -625.78, steps = 200\n",
      "23:31:09 [DEBUG] train episode 114: reward = -500.83, steps = 200\n",
      "23:31:43 [DEBUG] train episode 115: reward = -487.49, steps = 200\n",
      "23:32:15 [DEBUG] train episode 116: reward = -632.51, steps = 200\n",
      "23:32:49 [DEBUG] train episode 117: reward = -625.55, steps = 200\n",
      "23:33:22 [DEBUG] train episode 118: reward = -616.66, steps = 200\n",
      "23:33:55 [DEBUG] train episode 119: reward = -618.17, steps = 200\n",
      "23:34:28 [DEBUG] train episode 120: reward = -737.08, steps = 200\n",
      "23:35:02 [DEBUG] train episode 121: reward = -492.15, steps = 200\n",
      "23:35:35 [DEBUG] train episode 122: reward = -494.88, steps = 200\n",
      "23:36:10 [DEBUG] train episode 123: reward = -636.74, steps = 200\n",
      "23:36:44 [DEBUG] train episode 124: reward = -374.99, steps = 200\n",
      "23:37:18 [DEBUG] train episode 125: reward = -370.64, steps = 200\n",
      "23:37:53 [DEBUG] train episode 126: reward = -250.61, steps = 200\n",
      "23:38:31 [DEBUG] train episode 127: reward = -371.19, steps = 200\n",
      "23:39:06 [DEBUG] train episode 128: reward = -491.10, steps = 200\n",
      "23:39:39 [DEBUG] train episode 129: reward = -479.14, steps = 200\n",
      "23:40:14 [DEBUG] train episode 130: reward = -346.20, steps = 200\n",
      "23:40:47 [DEBUG] train episode 131: reward = -556.07, steps = 200\n",
      "23:41:22 [DEBUG] train episode 132: reward = -617.03, steps = 200\n",
      "23:41:55 [DEBUG] train episode 133: reward = -489.44, steps = 200\n",
      "23:42:35 [DEBUG] train episode 134: reward = -615.86, steps = 200\n",
      "23:43:15 [DEBUG] train episode 135: reward = -419.48, steps = 200\n",
      "23:43:55 [DEBUG] train episode 136: reward = -250.64, steps = 200\n",
      "23:44:30 [DEBUG] train episode 137: reward = -363.71, steps = 200\n",
      "23:45:04 [DEBUG] train episode 138: reward = -247.18, steps = 200\n",
      "23:45:39 [DEBUG] train episode 139: reward = -493.37, steps = 200\n",
      "23:46:17 [DEBUG] train episode 140: reward = -134.23, steps = 200\n",
      "23:46:52 [DEBUG] train episode 141: reward = -248.73, steps = 200\n",
      "23:47:34 [DEBUG] train episode 142: reward = -322.34, steps = 200\n",
      "23:48:17 [DEBUG] train episode 143: reward = -137.95, steps = 200\n",
      "23:48:54 [DEBUG] train episode 144: reward = -142.42, steps = 200\n",
      "23:49:32 [DEBUG] train episode 145: reward = -135.50, steps = 200\n",
      "23:50:10 [DEBUG] train episode 146: reward = -135.85, steps = 200\n",
      "23:50:46 [DEBUG] train episode 147: reward = -129.90, steps = 200\n",
      "23:51:21 [DEBUG] train episode 148: reward = -135.97, steps = 200\n",
      "23:51:58 [DEBUG] train episode 149: reward = -234.83, steps = 200\n",
      "23:52:35 [DEBUG] train episode 150: reward = -252.52, steps = 200\n",
      "23:53:14 [DEBUG] train episode 151: reward = -249.02, steps = 200\n",
      "23:53:51 [DEBUG] train episode 152: reward = -8.71, steps = 200\n",
      "23:54:26 [DEBUG] train episode 153: reward = -123.60, steps = 200\n",
      "23:55:00 [DEBUG] train episode 154: reward = -131.90, steps = 200\n",
      "23:55:33 [DEBUG] train episode 155: reward = -260.60, steps = 200\n",
      "23:56:07 [DEBUG] train episode 156: reward = -125.73, steps = 200\n",
      "23:56:40 [DEBUG] train episode 157: reward = -136.42, steps = 200\n",
      "23:57:14 [DEBUG] train episode 158: reward = -126.93, steps = 200\n",
      "23:57:47 [DEBUG] train episode 159: reward = -134.05, steps = 200\n",
      "23:58:21 [DEBUG] train episode 160: reward = -244.43, steps = 200\n",
      "23:58:55 [DEBUG] train episode 161: reward = -133.55, steps = 200\n",
      "23:59:28 [DEBUG] train episode 162: reward = -362.12, steps = 200\n",
      "00:00:02 [DEBUG] train episode 163: reward = -131.61, steps = 200\n",
      "00:00:36 [DEBUG] train episode 164: reward = -135.13, steps = 200\n",
      "00:01:09 [DEBUG] train episode 165: reward = -134.04, steps = 200\n",
      "00:01:43 [DEBUG] train episode 166: reward = -138.63, steps = 200\n",
      "00:02:17 [DEBUG] train episode 167: reward = -373.00, steps = 200\n",
      "00:02:50 [DEBUG] train episode 168: reward = -142.16, steps = 200\n",
      "00:03:24 [DEBUG] train episode 169: reward = -139.56, steps = 200\n",
      "00:03:58 [DEBUG] train episode 170: reward = -245.38, steps = 200\n",
      "00:04:31 [DEBUG] train episode 171: reward = -146.46, steps = 200\n",
      "00:05:04 [DEBUG] train episode 172: reward = -278.09, steps = 200\n",
      "00:05:37 [DEBUG] train episode 173: reward = -483.60, steps = 200\n",
      "00:06:11 [DEBUG] train episode 174: reward = -144.99, steps = 200\n",
      "00:06:42 [DEBUG] train episode 175: reward = -252.48, steps = 200\n",
      "00:07:15 [DEBUG] train episode 176: reward = -24.26, steps = 200\n",
      "00:07:49 [DEBUG] train episode 177: reward = -269.18, steps = 200\n",
      "00:08:23 [DEBUG] train episode 178: reward = -140.31, steps = 200\n",
      "00:08:57 [DEBUG] train episode 179: reward = -258.38, steps = 200\n",
      "00:09:31 [DEBUG] train episode 180: reward = -385.12, steps = 200\n",
      "00:10:05 [DEBUG] train episode 181: reward = -139.21, steps = 200\n",
      "00:10:39 [DEBUG] train episode 182: reward = -137.42, steps = 200\n",
      "00:11:12 [DEBUG] train episode 183: reward = -139.26, steps = 200\n",
      "00:11:46 [DEBUG] train episode 184: reward = -251.88, steps = 200\n",
      "00:12:19 [DEBUG] train episode 185: reward = -151.85, steps = 200\n",
      "00:12:53 [DEBUG] train episode 186: reward = -136.89, steps = 200\n",
      "00:13:26 [DEBUG] train episode 187: reward = -22.10, steps = 200\n",
      "00:13:59 [DEBUG] train episode 188: reward = -254.05, steps = 200\n",
      "00:14:33 [DEBUG] train episode 189: reward = -21.10, steps = 200\n",
      "00:15:05 [DEBUG] train episode 190: reward = -130.68, steps = 200\n",
      "00:15:39 [DEBUG] train episode 191: reward = -241.49, steps = 200\n",
      "00:16:13 [DEBUG] train episode 192: reward = -267.09, steps = 200\n",
      "00:16:46 [DEBUG] train episode 193: reward = -486.91, steps = 200\n",
      "00:17:19 [DEBUG] train episode 194: reward = -145.43, steps = 200\n",
      "00:17:52 [DEBUG] train episode 195: reward = -270.20, steps = 200\n",
      "00:18:25 [DEBUG] train episode 196: reward = -277.29, steps = 200\n",
      "00:18:59 [DEBUG] train episode 197: reward = -256.98, steps = 200\n",
      "00:19:32 [DEBUG] train episode 198: reward = -275.76, steps = 200\n",
      "00:20:04 [DEBUG] train episode 199: reward = -279.07, steps = 200\n",
      "00:20:38 [DEBUG] train episode 200: reward = -302.62, steps = 200\n",
      "00:21:11 [DEBUG] train episode 201: reward = -274.25, steps = 200\n",
      "00:21:44 [DEBUG] train episode 202: reward = -160.14, steps = 200\n",
      "00:22:18 [DEBUG] train episode 203: reward = -141.26, steps = 200\n",
      "00:22:51 [DEBUG] train episode 204: reward = -141.68, steps = 200\n",
      "00:23:24 [DEBUG] train episode 205: reward = -152.61, steps = 200\n",
      "00:23:57 [DEBUG] train episode 206: reward = -151.53, steps = 200\n",
      "00:24:31 [DEBUG] train episode 207: reward = -141.90, steps = 200\n",
      "00:25:04 [DEBUG] train episode 208: reward = -137.09, steps = 200\n",
      "00:25:37 [DEBUG] train episode 209: reward = -17.85, steps = 200\n",
      "00:26:11 [DEBUG] train episode 210: reward = -145.22, steps = 200\n",
      "00:26:46 [DEBUG] train episode 211: reward = -134.22, steps = 200\n",
      "00:27:21 [DEBUG] train episode 212: reward = -133.58, steps = 200\n",
      "00:27:54 [DEBUG] train episode 213: reward = -15.32, steps = 200\n",
      "00:27:54 [INFO] ==== test ====\n",
      "00:27:54 [DEBUG] test episode 0: reward = -376.42, steps = 200\n",
      "00:27:54 [DEBUG] test episode 1: reward = -16.15, steps = 200\n",
      "00:27:54 [DEBUG] test episode 2: reward = -17.46, steps = 200\n",
      "00:27:54 [DEBUG] test episode 3: reward = -132.63, steps = 200\n",
      "00:27:54 [DEBUG] test episode 4: reward = -252.30, steps = 200\n",
      "00:27:55 [DEBUG] test episode 5: reward = -285.80, steps = 200\n",
      "00:27:55 [DEBUG] test episode 6: reward = -133.77, steps = 200\n",
      "00:27:55 [DEBUG] test episode 7: reward = -144.74, steps = 200\n",
      "00:27:55 [DEBUG] test episode 8: reward = -16.54, steps = 200\n",
      "00:27:55 [DEBUG] test episode 9: reward = -255.91, steps = 200\n",
      "00:27:55 [DEBUG] test episode 10: reward = -250.78, steps = 200\n",
      "00:27:55 [DEBUG] test episode 11: reward = -141.01, steps = 200\n",
      "00:27:55 [DEBUG] test episode 12: reward = -361.69, steps = 200\n",
      "00:27:55 [DEBUG] test episode 13: reward = -138.41, steps = 200\n",
      "00:27:56 [DEBUG] test episode 14: reward = -256.61, steps = 200\n",
      "00:27:56 [DEBUG] test episode 15: reward = -18.99, steps = 200\n",
      "00:27:56 [DEBUG] test episode 16: reward = -128.23, steps = 200\n",
      "00:27:56 [DEBUG] test episode 17: reward = -137.33, steps = 200\n",
      "00:27:56 [DEBUG] test episode 18: reward = -136.54, steps = 200\n",
      "00:27:56 [DEBUG] test episode 19: reward = -17.40, steps = 200\n",
      "00:27:56 [DEBUG] test episode 20: reward = -269.00, steps = 200\n",
      "00:27:56 [DEBUG] test episode 21: reward = -362.52, steps = 200\n",
      "00:27:57 [DEBUG] test episode 22: reward = -140.55, steps = 200\n",
      "00:27:57 [DEBUG] test episode 23: reward = -144.08, steps = 200\n",
      "00:27:57 [DEBUG] test episode 24: reward = -132.10, steps = 200\n",
      "00:27:57 [DEBUG] test episode 25: reward = -128.16, steps = 200\n",
      "00:27:57 [DEBUG] test episode 26: reward = -144.42, steps = 200\n",
      "00:27:57 [DEBUG] test episode 27: reward = -348.61, steps = 200\n",
      "00:27:57 [DEBUG] test episode 28: reward = -144.68, steps = 200\n",
      "00:27:57 [DEBUG] test episode 29: reward = -135.66, steps = 200\n",
      "00:27:57 [DEBUG] test episode 30: reward = -395.87, steps = 200\n",
      "00:27:58 [DEBUG] test episode 31: reward = -17.68, steps = 200\n",
      "00:27:58 [DEBUG] test episode 32: reward = -258.45, steps = 200\n",
      "00:27:58 [DEBUG] test episode 33: reward = -241.36, steps = 200\n",
      "00:27:58 [DEBUG] test episode 34: reward = -345.05, steps = 200\n",
      "00:27:58 [DEBUG] test episode 35: reward = -144.70, steps = 200\n",
      "00:27:58 [DEBUG] test episode 36: reward = -135.11, steps = 200\n",
      "00:27:58 [DEBUG] test episode 37: reward = -251.47, steps = 200\n",
      "00:27:58 [DEBUG] test episode 38: reward = -138.08, steps = 200\n",
      "00:27:58 [DEBUG] test episode 39: reward = -303.34, steps = 200\n",
      "00:27:59 [DEBUG] test episode 40: reward = -144.13, steps = 200\n",
      "00:27:59 [DEBUG] test episode 41: reward = -254.81, steps = 200\n",
      "00:27:59 [DEBUG] test episode 42: reward = -129.43, steps = 200\n",
      "00:27:59 [DEBUG] test episode 43: reward = -136.34, steps = 200\n",
      "00:27:59 [DEBUG] test episode 44: reward = -135.45, steps = 200\n",
      "00:27:59 [DEBUG] test episode 45: reward = -252.17, steps = 200\n",
      "00:27:59 [DEBUG] test episode 46: reward = -135.08, steps = 200\n",
      "00:27:59 [DEBUG] test episode 47: reward = -247.05, steps = 200\n",
      "00:27:59 [DEBUG] test episode 48: reward = -144.06, steps = 200\n",
      "00:28:00 [DEBUG] test episode 49: reward = -16.59, steps = 200\n",
      "00:28:00 [DEBUG] test episode 50: reward = -136.88, steps = 200\n",
      "00:28:00 [DEBUG] test episode 51: reward = -131.04, steps = 200\n",
      "00:28:00 [DEBUG] test episode 52: reward = -249.31, steps = 200\n",
      "00:28:00 [DEBUG] test episode 53: reward = -133.47, steps = 200\n",
      "00:28:00 [DEBUG] test episode 54: reward = -136.61, steps = 200\n",
      "00:28:00 [DEBUG] test episode 55: reward = -137.68, steps = 200\n",
      "00:28:00 [DEBUG] test episode 56: reward = -369.94, steps = 200\n",
      "00:28:00 [DEBUG] test episode 57: reward = -16.45, steps = 200\n",
      "00:28:01 [DEBUG] test episode 58: reward = -137.13, steps = 200\n",
      "00:28:01 [DEBUG] test episode 59: reward = -134.73, steps = 200\n",
      "00:28:01 [DEBUG] test episode 60: reward = -134.81, steps = 200\n",
      "00:28:01 [DEBUG] test episode 61: reward = -136.59, steps = 200\n",
      "00:28:01 [DEBUG] test episode 62: reward = -129.73, steps = 200\n",
      "00:28:01 [DEBUG] test episode 63: reward = -135.88, steps = 200\n",
      "00:28:01 [DEBUG] test episode 64: reward = -384.43, steps = 200\n",
      "00:28:01 [DEBUG] test episode 65: reward = -138.42, steps = 200\n",
      "00:28:01 [DEBUG] test episode 66: reward = -136.65, steps = 200\n",
      "00:28:02 [DEBUG] test episode 67: reward = -140.50, steps = 200\n",
      "00:28:02 [DEBUG] test episode 68: reward = -137.42, steps = 200\n",
      "00:28:02 [DEBUG] test episode 69: reward = -129.78, steps = 200\n",
      "00:28:02 [DEBUG] test episode 70: reward = -139.42, steps = 200\n",
      "00:28:02 [DEBUG] test episode 71: reward = -140.06, steps = 200\n",
      "00:28:02 [DEBUG] test episode 72: reward = -132.66, steps = 200\n",
      "00:28:02 [DEBUG] test episode 73: reward = -254.19, steps = 200\n",
      "00:28:02 [DEBUG] test episode 74: reward = -137.95, steps = 200\n",
      "00:28:02 [DEBUG] test episode 75: reward = -17.16, steps = 200\n",
      "00:28:03 [DEBUG] test episode 76: reward = -139.48, steps = 200\n",
      "00:28:03 [DEBUG] test episode 77: reward = -242.89, steps = 200\n",
      "00:28:03 [DEBUG] test episode 78: reward = -127.88, steps = 200\n",
      "00:28:03 [DEBUG] test episode 79: reward = -16.39, steps = 200\n",
      "00:28:03 [DEBUG] test episode 80: reward = -145.02, steps = 200\n",
      "00:28:03 [DEBUG] test episode 81: reward = -128.05, steps = 200\n",
      "00:28:03 [DEBUG] test episode 82: reward = -16.40, steps = 200\n",
      "00:28:03 [DEBUG] test episode 83: reward = -261.78, steps = 200\n",
      "00:28:03 [DEBUG] test episode 84: reward = -132.06, steps = 200\n",
      "00:28:04 [DEBUG] test episode 85: reward = -129.15, steps = 200\n",
      "00:28:04 [DEBUG] test episode 86: reward = -141.86, steps = 200\n",
      "00:28:04 [DEBUG] test episode 87: reward = -246.46, steps = 200\n",
      "00:28:04 [DEBUG] test episode 88: reward = -129.05, steps = 200\n",
      "00:28:04 [DEBUG] test episode 89: reward = -135.87, steps = 200\n",
      "00:28:04 [DEBUG] test episode 90: reward = -277.85, steps = 200\n",
      "00:28:04 [DEBUG] test episode 91: reward = -139.72, steps = 200\n",
      "00:28:04 [DEBUG] test episode 92: reward = -136.69, steps = 200\n",
      "00:28:04 [DEBUG] test episode 93: reward = -17.53, steps = 200\n",
      "00:28:05 [DEBUG] test episode 94: reward = -138.06, steps = 200\n",
      "00:28:05 [DEBUG] test episode 95: reward = -144.99, steps = 200\n",
      "00:28:05 [DEBUG] test episode 96: reward = -142.33, steps = 200\n",
      "00:28:05 [DEBUG] test episode 97: reward = -133.69, steps = 200\n",
      "00:28:05 [DEBUG] test episode 98: reward = -253.56, steps = 200\n",
      "00:28:05 [DEBUG] test episode 99: reward = -136.17, steps = 200\n",
      "00:28:05 [INFO] average episode reward = -165.17 ± 90.69\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABSiElEQVR4nO2deXxcZ3nvv8/sWmYk2ZIl70uczdmc2IRQCJQSSFhKgBIIZWuBplDocqG3Le3lXi4ttNCFAqWhoVBIy5aSUriQEAiBsIUkzkLsLE68JLZsWZItSxpJs897/zjnPXNmNCONPJZGsp7v56OPRuecOfPO0cz7O8/6ijEGRVEURbEEmj0ARVEUZXGhwqAoiqKUocKgKIqilKHCoCiKopShwqAoiqKUEWr2ABqlu7vbbNq0qdnDUBRFWVI88MADx40xPdX2LXlh2LRpE7t27Wr2MBRFUZYUIvJMrX3qSlIURVHKUGFQFEVRylBhUBRFUcpQYVAURVHKWHTCICLXiMheEdknIn/W7PEoiqIsNxaVMIhIEPg08FJgG/AGEdnW3FEpiqIsLxaVMACXA/uMMQeMMVngq8C1TR6ToijKsmKxCcNa4LDv7353WxkicoOI7BKRXcPDwws2OEVR6idfKPK1+w9RKGpr/6XGYhMGqbJt2qfKGHOTMWanMWZnT0/Vwj1FUZrMLw6M8Ke37ubn+483eyinDWMM4+ncgr7mvQdOsOfIWNm2yUyel37iJ9y2e2BeXnOxCUM/sN739zrgaJPGoihKA4ylnAn0yMlUk0cC/+u/d/PjJ0/Nu3BgeIKLPngHB49PcufjQ1z+4TsZmcye5hHW5s+/sZuP3bG3bNvAWIrHB8bJFYrz8pqLTRjuB84Wkc0iEgGuB77V5DEpinIKTGQcYTg62lxhKBYNX7r3EHc9MXRKz999ZIxkOs/+oQmeOTFJOlfk4PGJ0zzK6hhjODqaZnAsXbZ9wP17dUfLvLzuohIGY0weeA9wB/A4cIsx5tHmjkpRlFMhmc4DcLRiUltoJrJ5jIGJTP6Unm8n4WQm572nI6ML857GU3lSuQLHxiuEYdQKQ2xeXnfRNdEzxtwG3NbscSiK0hh2Ih4Ya67FYCfzifSpCYO1eCbSee89LZR77Kh77cZSOdK5ArFw0NsuAr2J+RGGRWUxKIpy5mAn4qMLdHc92zhO1WKw4x9P50m6gecjo1OnZ3CzcMxnbQ36rIZjY2m626NEQvMzhaswKIoyL9iJ+OhoCmOal7JqJ/NTF4aU9/zSe1oYsRsYKxcDb0xj6XlzI4EKg6Io80TSnUQz+SInpxY2xbNsHA1aDNYVlkz7YgwL5Eryu+EGkxnv8bGxlAqDoihLD79Pv5mZSbbu4FRiDKlswRO1ZDrvCz4vjBU0MJamPeqEgv2ZSQOj6XnLSAIVBkVR5omJTJ5EzJnUmikMjVgMR3137BO+GMNEJs/4KQaz58KxsTRnrWqnJRz0YgzJdI5kJq8Wg6IoS4+JdJ5zeuNAc4XBCsJEJk9xju05bFpoOCgk3aykuCt2C+FOOjqWYk1HjN5E1EtZtbGG1Z1qMShNwhjDnY8Nks4Vmj0UZYkxkcmzYWUrkVCgLIi60CR9LSym5vg5toK2pbudZMZxJZ3flwAcd9J8Yozh2JjjMupNxBgad2IMpeI2tRiUJvHgoZO84+Zd3PHosXl9nceOjvMnX/8lmXz1L26+UOQ7jwywb2j2itNHj47Rf7J2OuGB4Qme+zd3laX/KafG0HiaHz4xVPX/lkznSMTCrOtq4fY9x3jo0EkA7n5ymPd8+UGOjqb4158c4CO3Pc7xicy05+cKRQbGGvflJ30un7nGGWy9wNbedsamskxlC5zbtzBW0Hg6z1S2wOqOGL2JmGcx2ID0fArDoitwUxYXd+91+svY9LzhZIaeePS0v873Hxvkll39nNeX4G3P21y2b3Qqy3WfuYenhiZ4+cWr+fRvXjbjuf7wqw9z8doO/uH126vuf2pogiOjKQ4en5y3AqGlzkOHTvLXtz/BzW+7nFg4iDGG/9zVz7quFoaSGW7ZdZhcocjDh0fJFQw98Sj/8uYdXLahC3Dudq3b5S+vvZD33fJL3vDZX3Dv+6/i9t0DfPuRAb675xh517XzlfsOcccfPZ9DI1N84L/38K33PI/P/+wgf3vHXnoTUQShszXMK7ev4R3P20IkFGBoPM3ewSQXr+ukoyVc872UCUMmx+GRIms6WwgGnJ6djw+M8+MnhxmZyrKus4WXX7yGFW0RAA6PpOhuj7KyLeJlBW10raDTZTE44udM9CLOmPKFInc+NgjA6s4YwxMZju1J8+p//hmPHh0nILAqrsKgNIkfP+V0xhxKprn/6RFe9y/38P3/8QK2rmoH4JsPH2HnphWsbdDfae+CPv3DfbzuWeu9TAxwunQ+NTRBPBoqy+WuxVQmz8mp2k3OMvli2W9lOg8dGuW+gyM8fWKS8/oSHB5J8Se3PuLt39LTRnd7lDc+eyNXbFnBH//nI3z9gX5PGFK5AkUD7dEQz93azQdfeQHv/I8HOHxyiqFkhjUdMc5a1c6vX7KGdZ0t/Oa/3svDh0d5cjDJU0MTHBtP038yRVskyHO2rCQcDPDk0AQf++5eLl3fRTpX4Le/cD8Av/erZ/E/rz6Xz//saV61fQ0r250bl5vveZru9miZK+np41O860sP8HfXXcK1252O/u+75Zc8NjBOOCjkCobDJ1P8+cvO5+4nh/nvh4/w0gv7aI+GvPbhiViYtZ0t02IMk5k8H/3uE/zPq88lHqstVH4+c/d+/ub2JwC48uxu/ukNl3F8MsPvf/khHhsYp6s1zEVrOzg2liZbKHLw+CRvevZGLliTmLfiNlBhUGZgdCrLI/2jgGMpPHEsiTHw1GCSravamcjk+cOvPswfXXU2f3TVOQ291sBYms7WMCcms9xy/+Eyq8EGD8/ti0/rGVONbMHMmIFi4yUaN6lNyr02A6NpzutLcGLSuVv+nSs386xNK7jq/F4CgVKX/Bt/tJ/DIyX3nXXZtLuB2j7X7TE4nmYomebcvjj/9tuXA+VdWK17xqkZyNGbiPGP118KOC7Cl3/yp4ync95rdbdHOHh8kv3DE/zltx8jKPBbz3U+O1/4+dP0xmMUjSEUEPJF43YkNez3uSRHp7K85rK1/P11l/Cif7ibQyemSKZzvPtLD3JOb5y/fs1F/McvDnnHx2MhRxgqLIaHD49y8z3P8Lyt3bzkgr5Zr/Ej/aP83R17edF5q7h0Qyef+MFT7Pir75MvGrpaw/zj67fz0ov6iIaCvHhbL4/0j/HeF5/Dpu62Wc/dKCoMSk1+tu+Ed9c3lMx4X1r7hTjm3uWnso1PsMfG0ly+aQW/7B/lsYHxsn0T7h3fWT3tPNI/hjHGM7mrkS8WmcjUHlNGhWFW7DWy/+tRd/J+6UWrPavAz4aVbd5NBJSK26zl15tw7uIHxzMMjWe4YHWHd2xHS5h4NMSR0ZTnsrQ1A1ZYAOLRsLfPpoqevzpB/8kUz5xwhMJfBJbOFugfnSIeDdObiHFkNMWB45MAZTcYTlptGBFhbWcLA2Mp9g9PMpHJ894Xn0M8Fi4bR7srDHftLe/War8H9cau/uo7j9PdHuUfXredjtYwz93azW27B+hqi/CaS9d5YgqwcWUbn3zDpXWd93SgwqDUZNczI7SEgzxvazd7B5MMuJNEv2tC2y/x6Zhgj46luGLLCsbTbRx0v7wWe/e/paeNbKHIWCpHZ2uk5rly+SKTM1oMrispp66kWqRdN5t18Y25RV6dNXz5G1a0cPvuAfKFIqFgwPPr29TO7vYoIs75jk9kWJUoj1Otce/Aj/gsBn9qKJSsjwnXmmiPhtiwopU9RwY45FoQNnMHHKsnmcxTTDj++yOjKfYPT7jvy/ns2liIFbC1nS08PpD0zrdxZSuAV4/hvKcwazpbGE5myhrbWSurHqsWYN/QBNdc2EdHq3NNL93QxaVVRLcZaFaSUpPJTJ6OljCrO2MMjae99slWGKy/P9WgMEy4aYB9HS1s7p4uDMl0nmgowNouJ44xOD49g8VPbhZXks2gSdfIgFJKd79W/G3MpquGIG9Y0Uq+aLwJ13MluXf54WCA7vYojx0dp2hgVUUCw9quFvpP+oUh703+Fvt4IpNnIu2IxrquVk5O5XjctTKHkqVJOZUrkC8ajo6lvJx/60Kyd/VeLMSd+Nd0tnB8IsO+wSQA69zPXOU47GfRH/PyhGFs5s8nODdTI5PZhmNz84UKg1KTTL5IJBRgVTzGZLbgfansl9dOAukG77ytS2pNZ4zN3W2MTGa9O1Rw3BLxWMjLIJrJVDfGkCsWZ4kxqMUwG9YKtO7DUff/kahhMaxf4dxZ2zttu0iPf0LtTUTZ7S5R2VORUbOmM8ZTg0myrqViXUn+IG4kFCAaCpTcTNGQN3Hfc+AEUPpsFIvG+z/bjB+ASVfwpgtYSRgA7j04Qnd7lNaIs90/joTrSoLyWgZ7zepxJdnruqZzcWbFqTAoNcl6wuDc3Z1wlzM84tYIWDdDo64k+yXtS8TY3O1kOx08UbIaJtxJoDc+uzAUigZjnLFna2QdafB5dqwryb8eQCIW8lI8K9lQIQyVriSA3niMITcGUOlKWtvZ6qWu2udbq8BPPBYimSlVIFthODzijNOevzLjbEVbpCyLJ5nOM+nrlhr3LAbnM/bQ4VE2rCjdzVe6tDxh8GUmpTzRmT2N9ai30I5aDMoSI5svEg0Fyr7E67pavL70A6fJlWTPs6azhc3dzgTjXzpxIuMEIe04hpK1TXX/5FIrzpBWV9Ks2Enu2FiaYtEwOpWlq612XGd1RwvhoPgshvI7cYBVvpqRaq4kP2OpHBPZPPFopTCEvZ5F8ViYdV2t3r5gQBidcha0qfxMtkdD3rls3sKx8bQ3zrZIKcYAzmffip3/fQQDQks4SF9HDJFyiyHlWQyzu5Ks4KorSVly+F1JlmdtWgE4Xwh7Z9SoS8b2o1mViLJ+RSsBgYPD5RZDPBomFg7S0RKe0WLI+hZHr+VOsuNt1AV2JmPjMLmC4fhEhpNTuZqBZ3AmzHVdrSVhcC2GNt/E3ucThsoiybU+l4ptGGcM0+oB2qMhr/11eyxEd3uEqGsJbFvttKoYTmamCUM8FvLiCGe7NTiDY+maabVQco+BU7tgX19EiIQC9MZjVYXBv25DLY6Ozu8KbI0yb8IgIn8rIk+IyCMi8g0R6XS3bxKRlIg87P58xvecHSKyW0T2icgnZaacRGXe8SwG35d45yYna6J/JFWKMTR4531sPOV+wYNEQ0HWdbV6aYXgxBjsF3dVPDqjMOQLJYuh1pcz7RW4qcVQC38K8tGxNKOpHB0zZIKBM5Ee9lkM0VCgzH1jU1Y7W8NEQ8Gy567tdCbhtkiQNZ0x7466mitpIpMn6XZuFRHPnbRjo/PZHEqmp6VQJ2Jhzyq4ZF0n4FiqlWm10VDQEy2/MLRFg9PGs6YzVuZKSvtec7ZCzIHRND3zuAJbo8znqL4PXGiMuRh4Eni/b99+Y8x29+edvu03AjcAZ7s/18zj+JRZyBSKREJBOlvDhIOORl/uWgx7B5OeH7nROoaBsfLe8pu623jo0Cgf++4TjExmHbeBlw8fm9FUz9VhMZRiDGox1CKdL3gB26OjKcamsjNaDAAbV7Syf2jC+Z9lpscH7N1xpRsJHAsiFBDWdrUQj4U9K7K94hyOxVCesbTWdSdZa3ZwPOP9j6014bcYLl7fCbiupCqxEBuA9ruSQsEArZFgmWtsbVdrWVtu/+dptgC0P1NqMTJvwmCM+Z4xxn4zfwGsm+l4EVkNJIwx9xina9bNwKvma3zK7GRyBSLBACJCT3uUWDjAWT3tREMBHnjGaYgWDsqcLQZjDLfvHuDn+5x2GwOj6TIT/uK1HRwZTfHPP9rP9x495sUYwHE3DY2nSecKXosCP/6A80QmTzZfxBhDvlDkL7/9GIPuc6E8+JwrFJu6/GSj5ApFnnRTLOfK3mNJCkVDMp3jQ//vMVLZAulckS09ToXt0dEUJ6dydLXOLAzXX76eXMHw3lse5unjk2WTKJQCztV6/AQDwprOFtZ2thCPhRh0006nuZJiIU5OZUnnit6+dV0tiMBlGzsBZ1K2bp0tPe3eeezNxaaVrXS2hjk2lmYyO93lZd1afmEAR5QSvvFsXtlalmKbyhUIucH52SyGI6OpMvfZYmOh7Ji3Abf7/t4sIg+JyN0icqW7bS3Q7zum3902DRG5QUR2iciu4eHh+RmxQrZQJBp2PiI9iRhrOloIBIRtaxLc/aRz3TesaJ3TnXe+UORtX7ifd33pQT707ccAJ4tjjU8Y/seLz+Ge9/8a4ASabVYSOHedA+NpLvrgHbzk43fzY3ccY6kco1PZaZktV37sLr583yEOHJ/kcz89yN17h72MlVI6o+H5H/shX7nv8Cldp8XAfz3Yz8s+8RNOVOlSOhOD42le+okf8+1HjvKzfcf5/M8O8uChk6SyBXoTMeLREAePTzKent2VdMGaDv7XK87nR3uH+fn+E5ztrsVgmcliAPib11zEH199LvFYCKvRlVZHIhb2kg/svjdfsZEPXXshfYkY4aAwlMx4Vuz5q50xdLWWqpd7EzH6EjHHlZSeHiTf3N3mZMFV+P8TLeGy8bz+8g0ERfinu54CHGGw7qdj42mMMXzx50+zx03RtRhjGBhNs2aRZiRBg5XPInInUK0pyF8YY77pHvMXQB74krtvANhgjDkhIjuA/xaRC4Bq8YSqt3DGmJuAmwB27ty5dG/zFpCHDp3kX39ykE9cv51QsL77gWy+SNQ99rU71nk+1L997cW86tM/ZyKTZ3N3O/cePFH3OB4fSPLDvcN0tYY5Mppi0l0Jq8/3JQkGhNUdLXS2hjk0MkW+aLwv9Xluy+OXX7SaBw+N8odffYiH/vdL+PP/2s14OscHXrHNO8+hE5MMjmc4OOw0ggPHirDtHmyMIVsoMjCW5pmR8sK6pcSB45Pki4b+kymviVw9HBtLUzSwf3jScxVNZPJk8gVawkHO7Ytz/9MjGFO76tnPm6/YSF8ixuqOFratSZTtW9EaIREL1ez18ytbu4FS6wvn8XRXkhUNO5mfvzrB+W7geVU8VmYVvvHZG3ntjnWsSsRKNxfxGH0dznETmTzhoHguJ4B3vuAsXrV97bTU3A+8YltZF9e1nS284fL1fOneQ7zzBWeRzhXobA2TiIUYHE/z7UcG+D/fepTeRJTv/uHzvayu0akcqVxhUbuSGhIGY8xVM+0XkbcCrwBe5LqHMMZkgIz7+AER2Q+cg2Mh+N1N64CjjYxPKXHvwRG+s3uAP3vpeWVBtZnI5EsWw5uv2Oht37oqzqffeBlfve8QG1a0cveTQ7VOMQ3re71iy0pu33PMW1+hWm/5VfGoVwVtJ4hfv3gNv3beKuKxMJ/+4T7+9o69pHMFDh6fxFAeY9jvZjZZnzQ4KayVBW727/Rp6PnULKzrYmAszSXr63/eiFvRfHQ0Rcp1q0yknWsUCwc5b3XcayDX1Ta7MIhIzQZygYDwnT+4ku5ZhMt/V17pSpppH1hXYykrqbM1zFk9TlB6XVcrPfEoiZYQqzti7Dky7lmj/jyXeCxc9dwvOKdn2ra3P28LX7znGX785DCprCOmm7vb+O+HjvCtXx7lrJ42Do1M8ftfeYjfef4Wnn92t9c5YFm6kkTkGuBPgVcaY6Z823tEJOg+3oITZD5gjBkAkiJyhZuN9Bbgm/M1vuWG9b37WwbU85xIDeviBef0cOObdtAaCZErmKr+/mrYPjLb3QDgg+7iLdWFIcYBt7eNtRgCAfG+tN3tzh3Y8YkMxycyZPMFcr6sJNsXJ5nJlZZ3zOan1TFYC6LReoxmMuAJw9zWCDg5WRIG2/JkMpsnlSsQCwc8Swugs2VmV1I9rF/RSkskOOMxlQ3rau2rdDMB9LRHvR5G4KS+Wt7+vM1874+ej4jQm4hxYjLDyalsWXxhrti4yXjauWYt4SB//7pL2L6hi2y+yD+/cQcfeMU27j14grd+/j4+ddc+vvnwEYIB4SI3O2oxMp9N9P4JiALfd9X4F24G0vOBD4lIHigA7zTGjLjPeRfwBaAFJyZxe+VJlVPDCkM9fVwsmXxh1nS6mGtRpHOFur5gg+NpAgIXrXO6az54aBSoXgG6Kh7lp1O2tcL0O7iVbc6XcjiZYWQySzQcIO+zGA74LAabfTKZyfsshfLspKWcpWQthnrWq/Az4hMGez1OTuYoFA0t4aDnowe8Zm/zjRX+gDjpq378sYBqwtDZGuaR/pwXY/ALQyQUIBJyxG11RwxjnM9IZZB8LsTCQSKhAONpxz0UiwTZuirOzW+7nFyhSDgY4Ny+ONftWM8ffe0hbvzRfgICr7h49aItboN5FAZjzNYa228Fbq2xbxdw4XyNaTljC7/q7fwIto5h5rs7e/eXqlMYjo2l6W6Pst5NMXzQzW7q7aiSwuiruK42CXS7QcwDw45/PZMrVi1ws711ACYzhZLFYAUhv7QtBrs2MDDntZVtc7yjY2nvegxPOOeIhYOc4wsg12qgd7qx/+tKFw9QlhVUzd2TiIXdSdp5L7WsExtY3j88wcXrOqoeUy+JWIjxVJ6060qyhH3WdkskyF+8bBs/fOJuUoUi73zBWQ295nyjbbeXCZ4rqU5hyBeKFA2zWwyucNTbd2gwmaGvI1bWUsAWt1XiT2usdle30g3m7XXTNLOFYpkryZJM57xCJifGUB58Xuq9k0YmsyXhn7PF4Fhk2XzRu2k4nnTEIhYOEo+FWb+ihcMjqbqCz6cD2+K62sRf5maq8pnoaAkzlS14MaVojc+vtVAz+WJDFoMz3jBJ12LwC0MlG1a28qcvPY+joykvWL5YUWFYJtgUzXotBnt8rS+WJeq5kupzwwyOpVm/opVwMEBPe5QhVyiq4U9rrGoxuEHMJ465wpAveq4kEbzsFX/wecIXfJ7+e2kKg7US2iJBBsZPLcbg57ib8mrXGTivz1kMp1Zn1dONdRtW+5/P5kqyYxwcz9ASDtZc0Mn/mWskxmDHkbQxhlniJ2+vWM98sbI467GV005ujneU1sKYzWKwd0j1TqrHxtP0uW4jm65Xq8OkXxiq3dW1RIK0RYI84fbiz+aL3vv0uxxs/36Ak7523pWWwlJ1Jdn/6SXrOxkcy1CsMxEAnKykygm2JAzO//4l23p53tbump1VTzdxz2KY/j+32yLBgCdcfjo8YUjPOEknYiHvs1vtdeY2Xsd9ZTO5zgRUGJYJdqKvd9lB65qYPfhcvzCkcwXGUjmvmZotaquWkQTljdYqs1Ms3fGoV/CULxrP0vFX6U5lC97SlCPu2sXt0RAZtyp6qbfIGHD/p5du6CRbKHopqPVwcjLLhWtKPvZELMTxCef5duK8bud6/v3tzz6NI54Zf4xh2j7Xmqj1eUi0ONsHx9MzunVExPvcNexKagkx7H4GZ3rNpYQKwzKhJAyZulo/2Myd2YLPJWGYfVK1omTbL1tLoaYryT0uEgzUHMfKilbQU1mbvx5xfzsTiU3jPOFOevbOMpMvek31Tsfa1c3g2FiKUEC8Cd5vFX7qB0/xv7+5p+ZzT05l2dTd5k1oZ/fGvaB9s+5+bWxhphhDrbt8aykeG097Fk8tbAC6WsbbXIhHS9XYs73mUuHMeBfKrFgLIJUreAupz3y8M0nOJV11No75FuSB0qIotVoDtEdDTuOyGUz9ymIpuwaDtRi2ur1y7GvblhlWMDK54pIPPg+MpulNxLw1DfyZST96cpifPHW86vOKRcPJqRwr2yKs6YzRFgmWWW/NE4bak38wINOa2fmxgp9M52f199v3ajunnirxWMi78VKLQVlS+JvL1eNOsi6ZWgVulpaKhdBrsbt/jH1uwZm1ENZ6MYbaFaCr4tEZTf3K9g8TnjA4FsNWt/e+dY9Y7ASSzhe8ArelKAzGOG0wbKYXlJZKBed/PZ7KVX1uMp2nUDR0tUVYv6KVdV2tZZNxs+5+o6EAXa3hsvUb/LRHQ7UtBl+AfLZJ2l6vRmMMZa85ixgtFTQraZmQzRcJB4VcwTA4ni7LT6+Gl5U0y+RQT4whVyjyGzf+nILrwrIm/AvPW8VfvupCdrrtkquxKh6bcdETW/1ssRZDZ4UwVOIJQ67gucFSuQLGmJqZLIuN/pNTvOGzv+DwSIrX7lhHd5vTutpaDMYYhpKO67Da+7KxiBVtYT7wim2ksgW++fARb3+zLAYR4dt/cCUratRNdLVGatZU+HsZzTb+vo7T5EoqE9MzQxjUYlgmZApFbxnEysyksakcH/7OY2WTu7UworNYDF66ao31lcHpfJotFL1qWpunHgsHefMVG2fMdvn9F23lf7z4nJr7rSvJtjueyDjvYf2KFiJu1anF7xbzXEn5kiupaMpXgCsWDT/aO4QxhrueGOSKj/ygIasimc7xl99+7LTFMh4fSHJ4JMWfXHMu//eVFxAIOK0e7P93LJVzM7VM1RiQrXruao1wVk87F67tKJskm+kWWdvZUvPu+++uu4T/efW5VfdFQwHPyp3VYrAxhtOQlWRRV5KypMjmi95KV5WupB89OcRnf3KQhw6NMpRM87X7D9VtMdgvQiZXYGQyWzWwPeqmiP7OlZv56GsvntMd+ZVn9/Dibb019690LQZ79zflNoK7bud6vv/e55e1HfC7rDrcvj/pXKFsPYmh8QxXfOQHPPDMCL84cILf+rf7+WX/GI8eGefYeNpL5TwV7j0wwud+epD7nh6Z/eA6sNbR1Rf0ebn4qztinsXgXxt7PD3dnWRrGFb4Avh+f/tivfu9aF2Ht85CJSLiZSbN5tZ59uaVXLt9Ddsb7FmU8AnLmeJKUmFYJmTzBRKxsLtmcvnkVppI0vznrn7+9NbdnnhEgvVlJR08PsmzP3InP3py+voYYylnAnru1m5eecmaht+LH2sx2AC2nSxbwkE2rmwru5vz99cvuZKKZXfT+4YmODaeZt/QhDeZDo2nvfqH2dbynQn73HpThus9nz8G09cR84oYh3z/52pxButK8rtl/G6RpXr3a33+s42/ozXMJ66/tOEeUGoxKEuWXMEQDgp9idi06mfrehhOZry0TntnPFtWUjgYIBgQ9hwdJ1cwPHls+ipi1mLonIdeO1YQNnU7brLJTIGA4Lmn/BOdXxisKymdK5St/Wy7z6ZzRc9qOjmVZdSdRCcbEAbblmNwjm0rajFVZfWxNZ0tDIylMMaUCVC9FoPflTRb1ftixaasLpTFozEGZcmSzReJhAL0uguU+LFiMJTMeCJh8/3rmRxioQAH3HUVqjVx84RhHloqbFjZyq3veg7XbncW+5vM5sual0VDAW+96nJXkj/G4M/YyrjbC14tx8mpnFcgZ2MYp4Ktvp5LI8MZz+eOpdU3GfUlYqRzRUancuWupNR0QRuZyhJx1zK2WH97JBQgsECVzqcb+79dKLdOxxmYlaTCsEzIFhxh6EtEpwmDFYPB8bQ3aZ2o02IA58uQnMFNYifVznlq27xj4wrvC+msyFUas0hp/YYyi6EsK6k02dvxp3NFL/ZwcirrdSGdqKMGpBYTmVzZazTKZCZPWyRYNoFb8RsYS5etvVHNYhibytHRGi6L+bS7MYal7BKp15V0uiizGJaolVXJmfEulFlxFt0J0puIMZzMlK1b4MUYxn0Ww2T9FoO/KrmaxTA2lUWkeiXr6cJmoky6SzX6sV9cv8WQqJKuCqWAbZnFMJn1rJ6GXEmn2WKYzOSnNYDzahnGUwyNZ7zAaLUYw1gqV3a3CyVX0lKu4LXveaGEwR/jUYtBWVJ4rqREjKIpFXzlCkWGXevgyGjK225/12Mx+CeRak36RlM5ErHwvDZhswI2mSlMW9PaW+u3WowhXySTd+ISUGpLnskVvdjDyalcyWJoJPhshWEOiyXNeL5Mflrxn20zYi0GW8cxVkMYKt177Qs8qc4HVuxiCzRJh4IBb0Gh2CwtZJYKKgzLAGOMz5XkTI7WneEUQEFrJMihEW8F1jm7kizDE+XWCLgT0Dyv/mXHOZnNT6vWthZDr7vwj996ybiuJBsYtxZDOl+yJE5MZLyJtRFhsO62E5OZsrWpT5VqFkNPPEowIBwbSzM4nmH9ilaioUDVNiijU9UshlKNyVLFWoML6daJx8JEl3BcphIVhmWALdqKuhYDlNwZtn3CRWvLV7GyxU+ztcSA0l1SR0uYQtFMaz8xOjX9zvR0Y91ZxkBomivJee0VbREioQDRUMCzcmzw2Y7Pdsn0WwyHRqa8tR0acSVZi8GY8hqDU2UyU5jW5ycYEFbFoxwddSyGVfEoiZZw3a4ke+cbXcLCsNDBZ3BuPs4UNxLMozCIyAdF5IiIPOz+vMy37/0isk9E9orI1b7tO0Rkt7vvk7JUehMscrK+vkd2CU1rMQz4evn7yRcNkVCgrmI0e3d52YZO95zli8WMpnJ0zPOykH7LJlzFYgiI4x5pj4acdXqDAURKwWdr0dgme46LyblufqFLNljHYOMfc11prRqT2TxtkelVu30dMfYOjpPOFelNxJylJ6sEn8dTuWmL74SCAVrCQVqWdIxhYYPP4FgpS9n9Vsl8//c/bozZ7v7cBiAi24DrgQuAa4B/FhF7RW8EbgDOdn+umefxLQusMISDQneb42oY9CwG57d/3Vvrr683j93efe/Y2FV2TsvYVHbeLYaZhGFNRwurO1oQEdqiQaKu4MVCQa/yubL3TqYiW8kym8WQTOf43qPHqra9mMjk2biyDZiemTSezvGR2x7n6GjtFdiKRePVU9ixVFt9bHVHjD1HxhFxBL+jJeylq05m8vxw7xCFoiGZyVd18bXHQkvalWSr4RdqxTlwLYYlfM0qacZtwbXAV40xGWPMQWAfcLmIrAYSxph7jNNX4WbgVU0Y3xmHXQc5EnJSG1fFo14AdGAsTWsk6AUpY+EAa9w2EvULg/OF2LHRaYZ3bDxNoWg4MDzBVDbP6ELEGIJ+YSi3cn7vhWdx67t+BYC2SGnSi4UDXuVzZfVrxmcx+KkUhmQ6x5ibsZQvFHnXfzzIDf/+AJd/5E7uPXCi4tg8Z68qbwNuz/HWz9/HTT8+wA+eGKr5Hm+8ez/bP/R9fuWvf8CeI2NMZApVhWHTyjYiwQD/9IbLeNamFY4rybUYbn2wn9/+t/t5asgpRKx0JdltjS532Uwu37SCm968gx0buhbsNV996Vpe/6z1C/Z68818//ffIyJvAXYB7zPGnATWAr/wHdPvbsu5jyu3T0NEbsCxLNiwYcM8DPvMonKZzt5EzOdKclo298ZLi+f4l0+sBzvRXrA2QSQU4NuPDPAP33uSZCbPb1y2rmr2y+nGLwaVFkNrJESr63Jpi4a8eEEsHCSTdyyDRCxctk50Olcgkyu/A1wVj3opp5b3/9duRiazfPl3ruAf73yKn+47zu//2lY+99OD3LZ7gGdvWcltuwd44bmrmMjkWL+ilUgowKCvxuAzd+/nl4dHATg+Q+zh8MgUkVCAo2Np7n96hMlM3qs78PMHLzqbN16x0esTlYiFefr4JOBkngE87i6HWk0YPvobF1XdvlQIBISXXNC3oK9pCyzPFBqyGETkThHZU+XnWhy30FnAdmAA+Hv7tCqnMjNsn77RmJuMMTuNMTt7enoaeQtnFMYYnhpMcsv9h8tcGZWL7vS5wpBM5/jJU8e5YE0Hna1hJwaRiHrCUG8AcmVbhJ54lEQszOqOGA88c5KeRJTt6zu58/FBjGHeYwwi4r2/0AyZIX2JGN1xZywtkSATmTwZd61evyvAWdmt3B20rquFyWy5MOwbmuCgO+n+14P9XHX+Kt73knNZ29nCsfE0+4cn+L0vPcitD/aTzhWJR0P0JqJlFsPB45Ns6m5jRVtkxiZ9yUyetZ0thALCsfE0qVx1iyEWDpY1D0y0hLysJNuOY+8xp1K9mgDs2LiCratmbsuunNk0ZDEYY66q5zgR+SzwbffPfsBvc60Djrrb11XZrtTJjXfv52Pf3Qs4mTmvucy5nJWL7pzd2873HjvGH//nL0mm8/zOlZsRETZ3t3FWT3upT1KdFsPvvXArv/lsx3Jb3eG0fb7xjTv46b7j/OW3HwPmpx1GJdFgwKvXqMVfvepCL8Dcl4hxdDRNtlAkFnYWl7dLg2byRTI551zZfJGAwOrOFh47Ol52vqFkhol0nmLRMDyR8SZUp5ldhsNuCvBet4dUeyzE2s4WjpwsxRKOjaXpS8QIBcTLiqrGRDpPIhaiuz3KM8ed89azXnEi5mQlOf2TnPM/OVjblaQo85mVtNr356sBu/Dst4DrRSQqIptxgsz3GWMGgKSIXOFmI70F+OZ8je9MZO+xJN3tUUTg8Ehp4vHWVnAnzHe+4Cy2rmrnjkcHed7Wbi522w7/xzuezftfdr6X3jlby21LR0vYC6r+xcu2cfPbLufcvjg7N5Z8vPMdY4CSRVTpSvLT5Vo34Ezez5xw7vanWQxucz1b99HREiYRC5e5kjJ5p9V4tlCk/2SKXMF45+5NxBgaT3N01LlDtxNxezTE+q5WDp8s1YwMjmfoS8ToiUdntBgmMnnaYyF64lGedsddTywg0RImXzSkcgXPhWiFaiH+L8rSYz5jDB8Tke047qCngd8FMMY8KiK3AI8BeeDdxhhrs78L+ALQAtzu/ih1MpbK0dcRJSBwZLQ08VTGGNqiIT7zph38ydcf4Y99C57YSW2uMQY/F/mym7atSXgB3oUUhplcSX5Wd8S8dtoxX22DSKm+oa8jxqGRKbpaI7RHg2XBZ39b68ePOZaEJzqJGEPJjCcAVhjisRDruloZHM+QyRcIBwIMjqfp7YhRMIaHDo3WHO9EOs/KtlaioSD37HcC23UJgyv046m8Jww21rCQmTvK0mHehMEY8+YZ9n0Y+HCV7buAC+drTGc6427BUjgY8L74UCpw87tYtvS083U3U6eSeDQ07fhTIRwMcMm6Tu49OOItjDOfeBZDnePu6yj54WPhoBdEX9kWIZ0rEAsH2NzeRiggdLQ6mTqpXIF8oUgoGChLOX1iwJn47VKjvR0xCkXD7v4xAE+A2qNhb8Gko6Np2qMh8kVDXyJGNl+c2ZXkWgzhQMBbY7ta8LkSu3DNkdEUkxVptOpKUqqxdKtYlGnYStZKH3apjqG+f7fnSjoNLQV2bnLcSV0LYDHY8YbrtRh8vZP8rqTu9qiXrhoNB+hsDbsWgzPB2snV3wzPZvms8lkMAA+72UaW9liI9SuctSMOj0x54tLrupJSuULNWomJTJ54NOQFzwEv22ombI3Go0cdkbI9q2LhQFkDREWxLN1kZWUaY6m84wtvCfO9RwcpFg2BgHh9eep1DcVjp8diAPjt525m48o2Vrorrc0n9cQY/PR1+IUh4FkMPfEo+4cnyOSDRENBfvXcVZzXFy8JQyY/bSW8J6wrqd05p+3LVNlbyV8I1X8y5Yl2X0fMO/b4RGaai8gYU4ox+K5lPcHn89x1r+983KmROLc3zmMD42otKDVRi+EMwRjjtThY19lCtlD0ApmZ/HRX0kzYDpun426yuz3K63YuTOGPFb56XUn+NtxRnyupJx4lVzBMZfJEQwH+7rpLeMeVW7zr4l+i07bWeGZkikgw4Llt+nzWyKaVrd5jJ101Rjgo9J+c8qyOvkTMc0NVC0Cnc0UKRUNbNERPvHTuemIMK9ujbFjRys/3HQfgkvVOHKhzAdx7ytJEheEMIZ0rki0UScTCrOtyJqJ+N85QmZU0G9aVdDoshoUkMkdX0oq2iCcmsVDQa4K2yp14J7OFstYQdhL2C0NvR5QVrRGMceILtrfUyvao57K5zJed1R4LEQwIazpbOHwyxeB4moA4z7WB62pxBvua8WjIO84ZU33ifemGTi9N96K1nYDGF5TaLK1vvlIT2/KgoyXMWje42e/GGaoFn2fidLqSFpKIa+HU60oSEc+dFAsHvDbN9s4dysXU70oCp/6gNx6j23Xt+Cds2+UUYKfbKsQ28gOnWK7/5BTHxtL0xKOEggHPRTRc0Z0WSsLQHguVja8eVxLAdrdJYjwWYnO3k1qsGUlKLZbWN1+piV0vwAafAS8Anc3PMcYQta6kpfXxsO+vcqGemSgJQ5C2aIh4LFQ22fprOez2o6MpHj06xlAyQ29HqZLaLwxQWhjo4nUdhAJCezTkWRTrOls5PJLi2HjaczutaIsgUr0thm3Z3R4Ne6/jF5rZuNTtG9SbiHnvWS0GpRYafD5D8AtDWzREZ2vYq2WorGOYjaXqSrJCFgnW50qCUpwhFg7yjis3c9X5vQxPlLKN/CtyWWH4wDcfpVA0CPDCc1d5dRPdFQF2O+GvX9Fatnqcs62F4xMZnhoUr7NtKBhgRWuE4YkM6VyBf/3JAY6MprlgTYItPW3eGJzW4QHCgfraogOcvzpOJOgs1GQD4yoMSi1UGM4QbIdP+2X3p6zOVRi84PMpFLg1E6/A7ZQshgCrO1pY19XKbbsHvP1+i8HGGGzbjWy+SF9HSQwqLYbNPW10t0dJxEKs6Yx5ra8Brrmwj8/99CDHxtO8pKPX297dHuXBZ07y65/6KU8NTRCPhfjKfYf40LUXAHhWR088Si5ftZVYVaKhIL/57A1s6WmjNRLid5+/hRdv6539icqyRIXhDMFaDNZvvLqjxStyszGGeiuC2yJBXn7xaq7YsnIeRjp/eFlJcxCGc1bFiYUDXnUwlK9h7c/MsoHe9Sta+MfXX8rbvnA/F67pIF8cBaYLw3teuJU3XbEREeHtz9tSlrq6dVWcb73neXzktse5xtcJtCce5af7jrOuq4Uvvu1yisbw2/92f6mlhiva3e3RqquyzcQHX3mB9/j9Lzt/Ts9VlhcqDGcI/uAzQE88wi/7R4HSHW69bgcR4dO/edm8jHM+KdUx1O9KetWla7ny7O6ytE+/GFSKxG/9yiZesq2XHRu7eOgDLyYQEC/7q6fCldQWDXnnvebC6W2g169o5cY37Sjb9t6XnMMrt6/h2u1riIaCXk+jJwedbqjWnXX5phUMz9BXSVEaQYXhDMGzGNw7yp72KCcmMhSKhmyhuOTcQqdCdI4FbuBmD1X4/2tZDFB+120XfrfBfpsN1giXbejiMt8CM9bVtW/IEQabMaZ3/Mp8cubPFmcwP3lqmKdcF8NYKkd7NOT513viUYoGp/vnLK2ozxTmWvlcC78Y1JOZ9StnreTr73yO16X2dJKIhWiLBBmZzBIKyJLLFFOWJvopW8K875Zf8sm79gGlPkkWf7HUchOG0BxcSdXwT771TMQiws5NKxp6zZnOba2GNl+6q6LMJ2f+bHGGkisUGZ7IeCtyjadynpsBSqmTwxMZsoXlJQyn0i7cj7/aObYIFnhf7XaBrbeYTVEa5cyfLc5Qjk9kMAZv7eBxt4GeZZrFsAxiDKeSlVSNMouhzsWK5hNba+EXfkWZT5r/qVeq8rN9x3ndv9xD3k01rcR29hwaz2CMmeZK8iyGZeRKip42V1Kw6uNmYYVBLQZloTjzZ4slyv1Pj3DfwRFGJqf3zQG8Pv6pXIFkJj9NGNqiTtDyuLqS5ozfSogtBovBzXpqV4tBWSCa/6lXqmKrZG19QiVDvkVihsYz04QBoDseZTiZIZMvNuxeWQrMT/C5+RaDP/isKAvBvM0WIvI1EXnY/XlaRB52t28SkZRv32d8z9khIrtFZJ+IfFKWcQqGrUsYS1Vfzcu/etjTxydJ5Qp0tZX31+9pj3qupOWQ5hgJzq27ai1ESmmhi+G6eTEGFQZlgZjPNZ9fbx+LyN8DY77d+40x26s87UbgBuAXwG3ANcDt8zXGxYy1FGpZDIPjGUTAGPipuwDLWT3tZcf0xKM8NTRBNBRYHsHnU6h8rkU0FHCW9lwUwqBZScrCMu+feveu/3XAV2Y5bjWQMMbcY4wxwM3Aq+Z7fIsVazHU6oczOJ5mi9tX/ydPDQNOB00/PfHosgo+P2tTF6/fuZ5tqzsaPlc0HCQUkDk15JsvErEQL72wj+ectbR6VylLl4X41F8JDBpjnvJt2ywiD4nI3SJypbttLdDvO6bf3dY0/vHOJ/neo8ea8trjswjD0HiGs3raaQkH2T88SWskyPqu1rJjutujjKVyjKVyy0IYOlsjfPS1F3srsTVCLBxYFNYCOK6tG9+0gxedr91QlYWhoU++iNwpInuq/FzrO+wNlFsLA8AGY8ylwHuBL4tIAqhm/1ftKywiN4jILhHZNTw83MhbmJGv3HeIOx4dnLfzz4QnDOnqMYbBZJq+jlJv/bN7417vHoutZRhKZuatMvdMJRoKLoriNkVpBg05LY0xV820X0RCwGsAr4WkMSYDZNzHD4jIfuAcHAthne/p64CjNV73JuAmgJ07d9bflH6OFIqGopm308+IFYSxKhZDOldgdCpHbyLGqniMp09McX5ffNpxG91F6N/34nN48xUb53fAZxiLyWJQlIVmvqNZVwFPGGM8F5GI9AAjxpiCiGwBzgYOGGNGRCQpIlcA9wJvAT41z+ObkULRUCguvDDkC0Wvd381V9KQW9y2Kh5llWsxnFtFGJ6zZSX3vP/XvOClUj/RUJCoWgzKMmW+heF6pgednw98SETyQAF4pzFmxN33LuALQAtONlJTM5LyRUOhCRZD0uc+qsxK+tK9z3DrA47O9iZi3pKR1YRBRFQUTpFoSC0GZfkyr8JgjPmtKttuBW6tcfwu4ML5HNNcKBQNxSZYDH730XhFHcMt9x9m72CSvkSM8/riPH1iknBQOL8vsdDDPKM5pzdes+pcUc50NDF6BvJNciVZKyEg02MMQ8kML79oDX//uksAeP2z1vMrZ3VPK25TGsO/II+iLDfUVp6BZgWfrRis7mgpcyUVi4bhZMaLK4DjC9+6qn3aORRFUU4VFYYaGGOaFny27qO1XS1lwefRVI580bCqYtF5RVGU04kKQw2sHuSbGGNY39XKeDqPca2WIXfthVXxWM3nKoqiNIoKQw3yRWcdhGa4kqz7aP2KFgpFw2S2APjSVBNqMSiKMn+oMNTAupCa4UoaS+UIB8VLRbXupKGkIww97SoMiqLMHyoMNbAupGL1BdTmlfFUjkQs7K2vYC0Iz5WkFoOiKPOICkMNCgXXYmhSVlJHS5hEzBGGsSlXGMYztEdDtEY0y1hRlPlDhaEG+Sa6ksbTeeItfovByVIaTmY0I0lRlHlHhaEGNujcrDqGjpYwiRbHMijFGNJex1RFUZT5QoWhBk21GFI5ErFQyZXkCz6vSmiqqqIo84sKQw28GEMNYfjIbY/zf//fo6f9ddO5AodHpli/opVES5j2aIj7n3Z6DKorSVGUhUCFoQa2jqGWMDz4zEkePjzq/X14ZIrP/viAV4x2quw+Mka+aLhsQxfBgPDbz93E7XuOcf/TI0xlC+pKUhRl3lFhqIFXx1Bjok/nC+QKpVzWr9x3iA/f9jiDbhHaqfLgMycBuHRDJwDveN4W4rEQv3PzLgD61JWkKMo8o8JQg1Idg9Mv6aWf+Al3+NZ/zuSK5Asl0dg3NAHA0ycmG3rdhw6NsmFFK91uEVtHa5g/veY8Nq1s4w9edDYv3qbr/iqKMr9oQnwN/BZDNl/k8YFx9h5LcvUFfYBjMYSDJV21wvDMiUmu2LLylF7TGMODh07yK2eVP/9NV2zkTbo0p6IoC4QKQw0Kvspn607K+1xHmVzpcTZf5JmRKQCePjF1yq95ZDTFUDLDZRu7TvkciqIojaLCUAN/uqrNUMr5AtHpXIFgQADHfWSF5JkGXElPDCQBuGhtxymfQ1EUpVEaijGIyHUi8qiIFEVkZ8W+94vIPhHZKyJX+7bvEJHd7r5Pioi426Mi8jV3+70isqmRsTWK35VkM5TKLIZ80Qs+PzXouJF6E1GePn7qFoPtidTVqquxKYrSPBoNPu8BXgP82L9RRLYB1wMXANcA/ywiQXf3jcANwNnuzzXu9rcDJ40xW4GPAx9tcGwN4U9XtSJhrQhjDJl8kWzeOWbf0AQi8MJzV/HMiclTTlmdzDitL9qiasgpitI8GhIGY8zjxpi9VXZdC3zVGJMxxhwE9gGXi8hqIGGMucc4s+fNwKt8z/mi+/jrwIusNdEM/G23SzEG53fGFQQrFPuGJ1jX1cJ5fXEmswWOT5zaIvITGWfdhbZocJYjFUVR5o/5SlddCxz2/d3vblvrPq7cXvYcY0weGANOLb3nNFDwpataQbBWhA08l1xJSbb2tLOxuw2AD/6/R/mzWx+Z82tOZvIEBFrCKgyKojSPWYVBRO4UkT1Vfq6d6WlVtpkZts/0nGpjukFEdonIruHh4ZnfwCnijzHYxznPYih4fxtjGJnM0puIsWmlIwzfeWSgrOahXiYyedoiIZpoKCmKosyelWSMueoUztsPrPf9vQ446m5fV2W7/zn9IhICOoCRGmO6CbgJYOfOnQ31oHjgmZOc09tO3G1YZ/FnJdnHNvic9qWq5ouGXKFIJBRgXVcLazpiZPJF0rnCnMcymclrfEFRlKYzX66kbwHXu5lGm3GCzPcZYwaApIhc4cYP3gJ80/ect7qPXwvcZRptPDQLmXyB62+6h6/df3jaPs+V5LcYiuUWAzjupGy+SCQYIBwM8NM//TVe96z1ZAtzX/ptMpvX+IKiKE2nodtTEXk18CmgB/iOiDxsjLnaGPOoiNwCPAbkgXcbY+xs+i7gC0ALcLv7A/A54N9FZB+OpXB9I2Orh1zBkCsYJtxsID9ldQwVFoMNPgPk8oZsoUg45GhsICBEggFyBUOxaAgE6ncLTWQKtKvFoChKk2loFjLGfAP4Ro19HwY+XGX7LuDCKtvTwHWNjGeu2Ik+V+XuvuAGmosGXx2DIxB+N1GmUCBXMER87TEirkjkikWigfotAHUlKYqyGFjWTfRsMDlXmO6x8jfIs8JRciWVhCSVdUTCigHgiUQ2Pzd3kgqDoiiLgWUtDNZFVG0C9y/padNTS8HnksUw6dYeREPTLYY5C0M2r64kRVGazrIWhtwMrqS8ry+SDSRXFriBM5kDZZ1WPWGYYwB6MlPQ4LOiKE1nWQtD3qtPqBZj8AlD3rqSplsMNnB9OlxJE+pKUhRlEbCshcEGmGeLMcxkMUy5rqSqwec6LIaxVI6rP/5jdvePkc0XaY+oMCiK0lyWtTBYQajm8qlmMVgLoyzGYF1JPovBupUydVgMz5yYZO9gkrufHAKgVS0GRVGazLIWBmsB5KpM4P4YQ65QHnwuizFYV5LPYojOIfg8kXaef2DYWcehXWMMiqI0mWUtDLli7eCzPytpJothKttYVtK4Kwz7jzvCoDEGRVGazbIWhsrmeEPJtLeWgj/GYC2EXBWLoWrweQ5ZSfb5B9w1o1UYFEVpNstaGOxEny0UGRhL8Zy/votfHHD69tnANPjWX6hS+TyVqZKuGqw/+DzhrtqWdM+jdQyKojSbZS0MXoyhUOR4MkuhaBhKpp19VYPPVWIMVSqfw3NIV63s09SmWUmKojSZZS0MBV8dg+2YaifzQpXgc66KxVAt+GxFop6spGS6XBjUYlAUpdksa2HwJvy8KRWxeau1VbEYfDEGO/lXizHMJSspWWkxaFaSoihNZlkLQ77MYnDjDa7lUKjSEsNropcrEHfv7G1WUjWLoa7gc7pSGNRiUBSluagw4EzgmQqLoWqBm89iaI85E/hkgy0xJjJ5z8IIBaQs7VVRFKUZLOtZyL8egxdjKJTXLEBpgi8aKBYN6VyBeKzCYvAHn+fQEiOZzrG521krui2q6z0ritJ8lrkwlOoYshW1Cv50Vb9LKFd0rAsbJG7UYkim86xf0UokFNDAs6Ioi4LlLQzFUksMO/lXVjn7t4EjJulcgfZoGPC33S7d6dvH9bqS4rEQvYkorRENPCuK0nyW9S2qrUvIForeYjwli2F65bPzHEMmX6QtGkQE0u7z/MFnESESCpCps/I5Hg2xOtFCwdeGQ1EUpVk0ZDGIyHUi8qiIFEVkp2/7i0XkARHZ7f7+Nd++H4nIXhF52P1Z5W6PisjXRGSfiNwrIpsaGVs95HwFbtnC7MFncOIS6VyBWChIOOBcvkgwMC02EA0GZrUYjDFMpPO0x0L871/fxv/59W2NvylFUZQGadRi2AO8BviXiu3HgV83xhwVkQuBO4C1vv1vNMbsqnjO24GTxpitInI98FHg9Q2Ob0Zs8LloSkHkzAwFblCyGKLhAOGgkC2Uxxcs4VBg1uBzOlckXzS0R8NcuLaj4fejKIpyOmjIYjDGPG6M2Vtl+0PGmKPun48CMRGJznK6a4Evuo+/DrxI5jlFxx9HsEHkXLWsJH/w2VoM4aCXfeSPL1gidVgMyYzTJ8mmviqKoiwGFiL4/BvAQ8aYjG/bv7lupA/4Jv+1wGEAY0weGANWVjuhiNwgIrtEZNfw8PApD8zfQdUWmlWLMVQGnzP5ItFQgJB1JVWxGCKh2YXBvmZChUFRlEXErMIgIneKyJ4qP9fW8dwLcFxCv+vb/EZjzEXAle7Pm+3hVU5RNRprjLnJGLPTGLOzp6dntmHUJO9LSZ3ITrcYrGT5J/ipbAFjIBYOEnEthZrCMIsraUI7qiqKsgiZdUYyxlx1KicWkXXAN4C3GGP2+853xP2dFJEvA5cDNwP9wHqgX0RCQAcwciqvXS9+d5G9ey810XOsgnSuWHX9hWgo4HMlVRGGelxJaRUGRVEWH/PiShKRTuA7wPuNMT/zbQ+JSLf7OAy8AieADfAt4K3u49cCdxkzv/mbed8dvY0xZH1ZSV6hWpXjouGgJwiRKsIQDgW8c9XCEwZ1JSmKsohoNF311SLSDzwH+I6I3OHueg+wFfhARVpqFLhDRB4BHgaOAJ91n/M5YKWI7APeC/xZI2Orh5w/xmCDz76spEjIKTjL1rAYQgHxHlfipKsWpm33Y8+ViIVP9S0oiqKcdhq6VTXGfAPHXVS5/a+Av6rxtB01zpUGrmtkPHPFH2Ce8CyGUoyhWvtsvzDY2EKtGMNUNj9tux+7epu6khRFWUws85YY0yd8f1aS19rC50pKupN5i8+VVDXGUEfw2bqStNW2oiiLiWUtDH5Xkhdj8K3vbC0Bv2VhJ/PWSMhzJVW1GOoIPtuW29WeryiK0iyW9YxUXt1cao9h91WbsMdTrsUQCZZcSTWCz7lZgs/j6bzXvltRFGWxsKyFoVrLCuv+KRhDNDS92+m4azH4XUmnajGMp3IkWjTwrCjK4mJZC0O+yh19Lj89XRXw3EY2xtAaCZZcSTViDJlZhGEslaNThUFRlEXG8haGYpHKbkylyudimSVgM5TGvRhDqVdSNYshGiqlqxpjOHh8ctoxY6kcHSoMiqIsMpa5MBhaw+XuIs+VVDBl2UYx9zgbY4hFgp6lMFtLjF8cGOGFf/cj9hwZKztmNJWlszVymt6NoijK6WF5C0PB0FKxapp/BbdoFYvBy0oKz+xKCgfFO9fxCad/4E/3HS87ZmxKLQZFURYfy1oYcoWiZwn4t8H0rCS/xRAJBggFfb2SqgafgxSNc550znEp3Xew1PqpUDSMp/MafFYUZdGxrIWhUDTT1lm2k3nBlAefrUgkM3nPyojM0Csp4quatkHo+58e8VJkbRBbg8+Koiw2lrUw5IqGlkipjsC2psjmixQK1S0GcFJVgZkL3HzCYC2GZDrP4wPjgBN4BtSVpCjKomNZC0O+UKQlXLoEbVG3aV6hOC3GEPMdZ62M8AwFblYYMoVCWdqqdSeNTqkwKIqyOFnWwuC4kqZbDLlC0emVVMtisMIwY4Gb22cpXyTjWgxrOmI8eOgkULIYOltVGBRFWVwsa2HIFYqeWwig3W1/nSsUyReLhAKC6y0qS121zwnX4UrKuUuBxsIBzumLs3/YqWdQV5KiKIuVZS0M+aIhFBSvi2rctRgyuSJFA8GAEHQn/7DvuJZ6XEnB0loOzhrRQbb2tHNgeIJC0TBqhUEtBkVRFhnLWxgKhlAg4FkDNsaQcl0/jsXgiEEwECAUcI5rrXAlVU1XrQg+x8IBtq5qJ5MvcuRkyiuUU4tBUZTFxvIWhmLRtQSsMDgWg80iCgYCnsUQCggh12KwcQlrQcyYruoGn6OhIFtXtQOwbzjJWCpHLByo2qhPURSlmSxvYSgYgoGSMFhXUipbshiCrsUQkNJxNhBt/662tKcVjYxrMURDAc7qcYRh/9Ako1NZOlu0HYaiKIuPRtd8vk5EHhWRoojs9G3fJCIp33rPn/Ht2yEiu0Vkn4h8UsSZeUUkKiJfc7ffKyKbGhlbPeQKRcLBgJdB1O6ujWBdSYGAEAz6LIaAtRhmz0qKTgs+B+lqi7CyLcK+oQltoKcoyqKlUYthD/Aa4MdV9u03xmx3f97p234jcANwtvtzjbv97cBJY8xW4OPARxsc26wUioZQQLwYQXvUmaj9MQZrMQSD1YTBBqZnDj5biwHgrFXt7BueYFT7JCmKskhpSBiMMY8bY/bWe7yIrAYSxph7jDEGuBl4lbv7WuCL7uOvAy+y1sR8kSsap+eRO7F7FkPWxhiEQFmMoboraaZ01Uy+4FkMAFtXtZcsBs1IUhRlETKfMYbNIvKQiNwtIle629YC/b5j+t1tdt9hAGNMHhgDVlY7sYjcICK7RGTX8PDwKQ8wX3BqFSpjDOkqFkNA/MFnZ5I/pzfOlu421ne1TDu3PSaVLZDJlyyGS9Z1MJbK8dTQhFoMiqIsSmYVBhG5U0T2VPm5doanDQAbjDGXAu8FviwiCaCaBWCXUZtpX/lGY24yxuw0xuzs6emZ7S1UpVg0FA2EguLFGGxWUipXshj8WUnhinTVravaueuPf5WV7dFp57fnmszkSeeKRN2WGq+5bB3n9sYpFI020FMUZVEy60r0xpir5npSY0wGyLiPHxCR/cA5OBbCOt+h64Cj7uN+YD3QLyIhoAMYYZ7Iu11Ow35XkpeV5PQ2CgUFVwucGINX4DbrZfPEY9K1GGKhkvvpr3/jIn7jxp+zKjFdUBRFUZrNvLiSRKRHRILu4y04QeYDxpgBICkiV7jxg7cA33Sf9i3gre7j1wJ3uXGIeSFfdCb/snTVyqwkKbmS/DGGlvDstQfRUIBQQKZZDACXbejiO79/JW989sbT94YURVFOE7Pf+s6AiLwa+BTQA3xHRB42xlwNPB/4kIjkgQLwTmOMvft/F/AFoAW43f0B+Bzw7yKyD8dSuL6Rsc2GtRj8WUmeKymbd/eVCtyCIl5vpMo1HKohIrRGgkxlC2RyhWmFbNvWJE7PG1EURTnNNCQMxphvAN+osv1W4NYaz9kFXFhlexq4rpHxzIV8oeRK8uoYZogxBAMBnyupvmrl9mjIsRjy5RaDoijKYmbZzlb5QimOEA4GEClN+Kmcu8/XKynka51RjysJoDUaYiKTJ+u2xFAURVkKLFthyPldScEA0VDA63nk1TEE/RbD9AK32WiLBDk5lQXKF/pRFEVZzCzb2apQsMIQcN1JAa+SuayOwRdj8ILP9QpDNMTIpCMMajEoirJUWLbCkCuWXEkr2sKsbI8iIkSCgVKMQfxtt/0WQ32hmdZISRjUYlAUZanQUPB5KZP3WQy//6Kz+a3nbgac/kf+lhhegVtwbumqAO3RICfdtZ3VYlAUZamwfIXBZzEkYmES7rKe4VDJYggFfU30Ak66aiRUSmGdjdZoiIIby6jWmltRFGUxsmxnq1K6avkkHwkGfBZDoCLGIHUHnqGU/gqlxnuKoiiLHbUYAuXaGA4GmPQK3Mqzkq46v5cVbfW3sfCLiFoMiqIsFZavMBRK6ap+IqEAR0aduMCKtkip7XZQeMkFfbzkgr66X6MtohaDoihLj2V7G+u1xAhWWgyOEJzXF2dNZwvW0xQMzP1StflcSWoxKIqyVFi2s1XOV/nsxy6w85JtvQBlMYa50hb1uZI0XVVRlCXCsp2tCsXqriTb9uLF2xyXkb+OYa6UuZI0XVVRlCXCshWGnK+OwU80FKAvEePCtU73U/9CPXOlVS0GRVGWIMs3+OxmJVWmq/7hi86hUDRIhaUQDM5dGMrSVdViUBRlibB8haFQPfj8nLPKl5luJMbgb52hFoOiKEuFZTtb5WvEGCrxr+A2V8qCz2oxKIqyRFi+wlAjK6mSQKCB4LPrSgr72ncriqIsdpatMJTWY5j5EgR9C/XMlVa3qE2tBUVRlhINCYOIXCcij4pIUUR2+ra/UUQe9v0URWS7u+9HIrLXt2+Vuz0qIl8TkX0icq+IbGpkbLNRKJRWaZsJazEETiHGEAoGiIUD2nJbUZQlRaMz1h7gNcCP/RuNMV8yxmw3xmwH3gw8bYx52HfIG+1+Y8yQu+3twEljzFbg48BHGxzbjJQqn2eJMbhXaDbLohZtkZBaDIqiLCkaEgZjzOPGmL2zHPYG4Ct1nO5a4Ivu468DLxI5hdv0OeCs2jbzJbCCcKoxgrZoSNthKIqypFiIdNXX40z6fv5NRArArcBfGWMMsBY4DGCMyYvIGLASOF55QhG5AbgBYMOGDac0qHdcuYV3XLll1uMCDcQYwOmwOs/6piiKclqZ9VZWRO4UkT1Vfion+2rPfTYwZYzZ49v8RmPMRcCV7s+b7eFVTmGqndcYc5MxZqcxZmdPT89sw2gIa1CcSowB1GJQFGXpMavFYIy5qoHzX0+FG8kYc8T9nRSRLwOXAzcD/cB6oF9EQkAHMNLAa58WAg20xAB4+UWrvSprRVGUpcC8uZJEJABcBzzfty0EdBpjjotIGHgFcKe7+1vAW4F7gNcCd7kupqYSbKCJHsDbnrf5dA5HURRl3mlIGETk1cCngB7gOyLysDHmanf384F+Y8wB31OiwB2uKARxROGz7r7PAf8uIvtwLIXrGxnb6SIYaCzGoCiKstRoSBiMMd8AvlFj34+AKyq2TQI7ahyfxrEwFhWNtN1WFEVZimhUdBYaaaKnKIqyFFFhmIXSegx6qRRFWR7obDcLoQbWY1AURVmKLNv1GOrl6gv6yBdN2aI7iqIoZzJqMczCpu423v3Crc0ehqIoyoKhwqAoiqKUocKgKIqilKHCoCiKopShwqAoiqKUocKgKIqilKHCoCiKopShwqAoiqKUocKgKIqilCGLYMmDhhCRYeCZU3x6N1WWDlU89PrURq/NzOj1qc1iuTYbjTFVl8Bc8sLQCCKyyxizs9njWKzo9amNXpuZ0etTm6VwbdSVpCiKopShwqAoiqKUsdyF4aZmD2CRo9enNnptZkavT20W/bVZ1jEGRVEUZTrL3WJQFEVRKlBhUBRFUcpYtsIgIteIyF4R2Scif9bs8TQbEXlaRHaLyMMissvdtkJEvi8iT7m/u5o9zoVCRD4vIkMisse3reb1EJH3u5+lvSJydXNGvTDUuDYfFJEj7ufnYRF5mW/fcro260XkhyLyuIg8KiJ/6G5fUp+dZSkMIhIEPg28FNgGvEFEtjV3VIuCFxpjtvtyrP8M+IEx5mzgB+7fy4UvANdUbKt6PdzPzvXABe5z/tn9jJ2pfIHp1wbg4+7nZ7sx5jZYltcmD7zPGHM+cAXwbvcaLKnPzrIUBuByYJ8x5oAxJgt8Fbi2yWNajFwLfNF9/EXgVc0bysJijPkxMFKxudb1uBb4qjEmY4w5COzD+YydkdS4NrVYbtdmwBjzoPs4CTwOrGWJfXaWqzCsBQ77/u53ty1nDPA9EXlARG5wt/UaYwbA+cADq5o2usVBreuhnyeH94jII66rybpKlu21EZFNwKXAvSyxz85yFQapsm255+0+1xhzGY577d0i8vxmD2gJoZ8nuBE4C9gODAB/725fltdGRNqBW4E/MsaMz3RolW1Nvz7LVRj6gfW+v9cBR5s0lkWBMeao+3sI+AaOOTsoIqsB3N9DzRvhoqDW9Vj2nydjzKAxpmCMKQKfpeQOWXbXRkTCOKLwJWPMf7mbl9RnZ7kKw/3A2SKyWUQiOMGfbzV5TE1DRNpEJG4fAy8B9uBck7e6h70V+GZzRrhoqHU9vgVcLyJREdkMnA3c14TxNQ076bm8GufzA8vs2oiIAJ8DHjfG/INv15L67ISaPYBmYIzJi8h7gDuAIPB5Y8yjTR5WM+kFvuF8pgkBXzbGfFdE7gduEZG3A4eA65o4xgVFRL4C/CrQLSL9wP8B/oYq18MY86iI3AI8hpOV8m5jTKEpA18AalybXxWR7ThukKeB34Xld22A5wJvBnaLyMPutj9niX12tCWGoiiKUsZydSUpiqIoNVBhUBRFUcpQYVAURVHKUGFQFEVRylBhUBRFUcpQYVAURVHKUGFQFEVRyvj/JNzqyskWancAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
